<?xml version="1.0" encoding="ascii"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
          "DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
  <title>pypln</title>
  <link rel="stylesheet" href="epydoc.css" type="text/css" />
  <script type="text/javascript" src="epydoc.js"></script>
</head>

<body bgcolor="white" text="black" link="blue" vlink="#204080"
      alink="#204080">
<!-- ==================== NAVIGATION BAR ==================== -->
<table class="navbar" border="0" width="100%" cellpadding="0"
       bgcolor="#a0c0ff" cellspacing="0">
  <tr valign="middle">
  <!-- Home link -->
      <th bgcolor="#70b0f0" class="navbar-select"
          >&nbsp;&nbsp;&nbsp;Home&nbsp;&nbsp;&nbsp;</th>

  <!-- Tree link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="module-tree.html">Trees</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Index link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="identifier-index.html">Indices</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Help link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="help.html">Help</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Project homepage -->
      <th class="navbar" align="right" width="100%">
        <table border="0" cellpadding="0" cellspacing="0">
          <tr><th class="navbar" align="center"
            ><a class="navbar" target="_top" href="http://code.google.com/p/pypln">Project Homepage</a></th>
          </tr></table></th>
  </tr>
</table>
<table width="100%" cellpadding="0" cellspacing="0">
  <tr valign="top">
    <td width="100%">
      <span class="breadcrumbs">
        Package&nbsp;pypln
      </span>
    </td>
    <td>
      <table cellpadding="0" cellspacing="0">
        <!-- hide/show private -->
        <tr><td align="right"><span class="options">[<a href="javascript:void(0);" class="privatelink"
    onclick="toggle_private();">hide&nbsp;private</a>]</span></td></tr>
      </table>
    </td>
  </tr>
</table>
<!-- ==================== PACKAGE DESCRIPTION ==================== -->
<h1 class="epydoc">Package pypln</h1><p class="nomargin-top"><span class="codelink"><a href="pypln-pysrc.html">source&nbsp;code</a></span></p>
<p>PyPLN -- Distributed text processing pipeline. Ports: 5557: task 
  ventilator push; worker pull port 5558: worker push port; Sink pull port 
  5559: Ventilators  Pub/Sub port 5560: Sinks PUB/SUB port for tasks 
  finished</p>

<!-- ==================== SUBMODULES ==================== -->
<a name="section-Submodules"></a>
<table class="summary" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr bgcolor="#70b0f0" class="table-header">
  <td colspan="2" class="table-header">
    <table border="0" cellpadding="0" cellspacing="0" width="100%">
      <tr valign="top">
        <td align="left"><span class="table-header">Submodules</span></td>
        <td align="right" valign="top"
         ><span class="options">[<a href="#section-Submodules"
         class="privatelink" onclick="toggle_private();"
         >hide private</a>]</span></td>
      </tr>
    </table>
  </td>
</tr>
  <tr><td class="summary">
  <ul class="nomargin">
    <li> <strong class="uidlink"><a href="pypln.Monitors-module.html">pypln.Monitors</a></strong>: <em class="summary">This package contains code implementing realtime monitoring tools 
        for the Pipeline</em>
    <ul>
    <li> <strong class="uidlink"><a href="pypln.Monitors.graphite_monitor-module.html">pypln.Monitors.graphite_monitor</a></strong>: <em class="summary">Graphite monitoring module.</em>    </li>
    </ul>
    </li>
    <li> <strong class="uidlink"><a href="pypln.servers-module.html">pypln.servers</a></strong>
    <ul>
    <li> <strong class="uidlink"><a href="pypln.servers.ventilator-module.html">pypln.servers.ventilator</a></strong>: <em class="summary">Task ventilator Binds PUSH socket to tcp://localhost:5557 Sends 
        batch of tasks to workers via that socket Adapted from 0MQ tutorial
        (&quot;The Guide&quot;)</em>    </li>
    </ul>
    </li>
    <li> <strong class="uidlink"><a href="pypln.sinks-module.html">pypln.sinks</a></strong>
    <ul>
    <li> <strong class="uidlink"><a href="pypln.sinks.base-module.html">pypln.sinks.base</a></strong>: <em class="summary">Base classes for sinks</em>    </li>
    <li> <strong class="uidlink"><a href="pypln.sinks.dummy_sink-module.html">pypln.sinks.dummy_sink</a></strong>: <em class="summary">Task sink Binds PULL socket to tcp://localhost:5558 Collects 
        results from workers via that socket</em>    </li>
    <li> <strong class="uidlink"><a href="pypln.sinks.raw_document_sink-module.html">pypln.sinks.raw_document_sink</a></strong>: <em class="summary">Document sink Binds PULL socket to tcp://localhost:5558 Collects 
        text from workers via that socket and stores in the Document 
        database</em>    </li>
    </ul>
    </li>
    <li> <strong class="uidlink"><a href="pypln.stores-module.html">pypln.stores</a></strong>: <em class="summary">This package contains code defining persistent stores which can be 
        used by the sinks to  save  data to, and by the servers to fetch 
        data for further processing</em>
    <ul>
    <li> <strong class="uidlink"><a href="pypln.stores.document-module.html">pypln.stores.document</a></strong>: <em class="summary">Define a Mongo Document database for storing raw text documents and
        metadata</em>    </li>
    </ul>
    </li>
    <li> <strong class="uidlink"><a href="pypln.test-module.html">pypln.test</a></strong>: <em class="summary">Test module containing tests for all modules of pypln</em>    </li>
    <li> <strong class="uidlink"><a href="pypln.workers-module.html">pypln.workers</a></strong>
    <ul>
    <li> <strong class="uidlink"><a href="pypln.workers.base-module.html">pypln.workers.base</a></strong>: <em class="summary">Base classes for workers</em>    </li>
    <li> <strong class="uidlink"><a href="pypln.workers.dummy_worker-module.html">pypln.workers.dummy_worker</a></strong>: <em class="summary">Task worker Connects PULL socket to tcp://localhost:5557 Collects 
        workloads from ventilator via that socket Connects PUSH socket to 
        tcp://localhost:5558 Sends results to sink via that socket</em>    </li>
    <li> <strong class="uidlink"><a href="pypln.workers.enc_detect_worker-module.html">pypln.workers.enc_detect_worker</a></strong>: <em class="summary">Character encoding detection worker</em>    </li>
    <li> <strong class="uidlink"><a href="pypln.workers.mime_detect_worker-module.html">pypln.workers.mime_detect_worker</a></strong>: <em class="summary">Mimetype detection worker</em>    </li>
    <li> <strong class="uidlink"><a href="pypln.workers.palavras_worker-module.html">pypln.workers.palavras_worker</a></strong>: <em class="summary">class: palavras_worker connect worker to palavras (bick), and 
        return annotation</em>    </li>
    <li> <strong class="uidlink"><a href="pypln.workers.pdfconv_worker-module.html">pypln.workers.pdfconv_worker</a></strong>: <em class="summary">PDF to text conversion worker</em>    </li>
    </ul>
    </li>
  </ul></td></tr>
</table>

<br />
<!-- ==================== VARIABLES ==================== -->
<a name="section-Variables"></a>
<table class="summary" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr bgcolor="#70b0f0" class="table-header">
  <td colspan="2" class="table-header">
    <table border="0" cellpadding="0" cellspacing="0" width="100%">
      <tr valign="top">
        <td align="left"><span class="table-header">Variables</span></td>
        <td align="right" valign="top"
         ><span class="options">[<a href="#section-Variables"
         class="privatelink" onclick="toggle_private();"
         >hide private</a>]</span></td>
      </tr>
    </table>
  </td>
</tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="pv"></a><span class="summary-name">pv</span> = <code title="[int(v) for v in version.pyzmq_version().split('.')]">[int(v) for v in version.pyzmq_version().split('.')]</code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="zv"></a><span class="summary-name">zv</span> = <code title="[int(v) for v in version.zmq_version().split('.')]">[int(v) for v in version.zmq_version().split('.')]</code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a href="pypln-module.html#TASK_TYPES" class="summary-name">TASK_TYPES</a> = <code title="{1: {'name': 'mimetype detection', 'worker': 'mime_detect_worker', 'si\
nk': 'mimetype_sink'}, 2: {'name': 'Encoding detection', 'worker': 'en\
c_detect_worker', 'sink': 'encoding_sink'}, 3: {'name': 'PDF to Text C\
onversion', 'worker': 'pdfconv_worker', 'sink': 'document_sink'},}">{1: {'name': 'mimetype detection', 'worker': 'mim<code class="variable-ellipsis">...</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a href="pypln-module.html#PORTS" class="summary-name">PORTS</a> = <code title="{'ventilator push': 5557, 'worker push': 5558, 'ventilator pub': 5559,\
 'sink pub': 5560,}">{'ventilator push': 5557, 'worker push': 5558, 'ventil<code class="variable-ellipsis">...</code></code>
    </td>
  </tr>
</table>
<!-- ==================== VARIABLES DETAILS ==================== -->
<a name="section-VariablesDetails"></a>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr bgcolor="#70b0f0" class="table-header">
  <td colspan="2" class="table-header">
    <table border="0" cellpadding="0" cellspacing="0" width="100%">
      <tr valign="top">
        <td align="left"><span class="table-header">Variables Details</span></td>
        <td align="right" valign="top"
         ><span class="options">[<a href="#section-VariablesDetails"
         class="privatelink" onclick="toggle_private();"
         >hide private</a>]</span></td>
      </tr>
    </table>
  </td>
</tr>
</table>
<a name="TASK_TYPES"></a>
<div>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr><td>
  <h3 class="epydoc">TASK_TYPES</h3>
  
  <dl class="fields">
  </dl>
  <dl class="fields">
    <dt>Value:</dt>
      <dd><table><tr><td><pre class="variable">
{1: {'name': 'mimetype detection', 'worker': 'mime_detect_worker', 'si<span class="variable-linewrap"><img src="crarr.png" alt="\" /></span>
nk': 'mimetype_sink'}, 2: {'name': 'Encoding detection', 'worker': 'en<span class="variable-linewrap"><img src="crarr.png" alt="\" /></span>
c_detect_worker', 'sink': 'encoding_sink'}, 3: {'name': 'PDF to Text C<span class="variable-linewrap"><img src="crarr.png" alt="\" /></span>
onversion', 'worker': 'pdfconv_worker', 'sink': 'document_sink'},}
</pre></td></tr></table>
</dd>
  </dl>
</td></tr></table>
</div>
<a name="PORTS"></a>
<div>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr><td>
  <h3 class="epydoc">PORTS</h3>
  
  <dl class="fields">
  </dl>
  <dl class="fields">
    <dt>Value:</dt>
      <dd><table><tr><td><pre class="variable">
{'ventilator push': 5557, 'worker push': 5558, 'ventilator pub': 5559,<span class="variable-linewrap"><img src="crarr.png" alt="\" /></span>
 'sink pub': 5560,}
</pre></td></tr></table>
</dd>
  </dl>
</td></tr></table>
</div>
<br />
<!-- ==================== NAVIGATION BAR ==================== -->
<table class="navbar" border="0" width="100%" cellpadding="0"
       bgcolor="#a0c0ff" cellspacing="0">
  <tr valign="middle">
  <!-- Home link -->
      <th bgcolor="#70b0f0" class="navbar-select"
          >&nbsp;&nbsp;&nbsp;Home&nbsp;&nbsp;&nbsp;</th>

  <!-- Tree link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="module-tree.html">Trees</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Index link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="identifier-index.html">Indices</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Help link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="help.html">Help</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Project homepage -->
      <th class="navbar" align="right" width="100%">
        <table border="0" cellpadding="0" cellspacing="0">
          <tr><th class="navbar" align="center"
            ><a class="navbar" target="_top" href="http://code.google.com/p/pypln">Project Homepage</a></th>
          </tr></table></th>
  </tr>
</table>
<table border="0" cellpadding="0" cellspacing="0" width="100%%">
  <tr>
    <td align="left" class="footer">
    Generated by Epydoc 3.0.1 on Tue Jul 26 17:36:27 2011
    </td>
    <td align="right" class="footer">
      <a target="mainFrame" href="http://epydoc.sourceforge.net"
        >http://epydoc.sourceforge.net</a>
    </td>
  </tr>
</table>

<script type="text/javascript">
  <!--
  // Private objects are initially displayed (because if
  // javascript is turned off then we want them to be
  // visible); but by default, we want to hide them.  So hide
  // them unless we have a cookie that says to show them.
  checkCookie();
  // -->
</script>
</body>
</html>
